import pymysql
import requests
from lxml import etree

db = pymysql.connect(user="root", password="544002", host="localhost", database="shiyu", port=3307, charset="utf8")
cursor = db.cursor()

headers = {
    "sec-ch-ua": "\"Google Chrome\";v=\"119\", \"Chromium\";v=\"119\", \"Not?A_Brand\";v=\"24\"",
    "Accept": "application/json, text/javascript, */*; q=0.01",
    "Referer": "https://www.bige3.cc/finish/",
    "X-Requested-With": "XMLHttpRequest",
    "sec-ch-ua-mobile": "?0",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
    "sec-ch-ua-platform": "\"Windows\""
}

for page in range(1, 20 + 1):
    print(page)
    # todo url格式化不带参数问题

    url = "https://www.bige3.cc/json"
    params = {
        "sortid": "0",
        "page": page
    }

    # todo  response = requests.get(url, headers=headers) 未加参数导致请求不成功
    response = requests.get(url, headers=headers, params=params).json()

    for respon in response:
        url_list = respon["url_list"]
        url_img = respon["url_img"]
        articlename = respon["articlename"]
        author = respon["author"]
        intro = respon["intro"]
        # print(articlename,"---",author,"---",intro,"---", url_list,"---",url_img)

        # todo  可以直接请求目标网页 不用selenium 进行点击操作

        chapter_url = "https://www.bige3.cc/" + url_list
        chapter_response = requests.get(chapter_url, headers=headers)
        chapter_result = chapter_response.content.decode()
        chapter_ = etree.HTML(chapter_result)
        result = chapter_.xpath("//div[@class='listmain']/dl//dd/a")
        for chapter in result:
            chapter_name = chapter.xpath("./text()")[0]
            chapter_url_ = "https://www.bige3.cc/" + chapter.xpath("./@href")[0]
            if "展开全部章节" not in chapter_name:
                article_ = requests.get(chapter_url_, headers=headers).content.decode()
                author_ = etree.HTML(article_).xpath("//div[@id='chaptercontent']/text()")[1:][:-2]
                author_content = ""
                for content in author_:
                    author_content = author_content + content
                print(articlename, "---", author, "---", chapter_name, "---", chapter_url_, "---", author_content)
                sql = "insert into story_bxg(articlename,author,chapter_name,chapter_url_,author_content) values(%s,%s,%s,%s,%s)"  # 注意此处与前一种形式的不同
                par = (articlename, author, str(chapter_name), chapter_url_, author_content)
                cursor = db.cursor()
                try:
                    cursor.execute(sql, par)
                    db.commit()  # 提交到数据库执行，一定要记提交哦
                    print("写入数据库成功")
                except Exception as e:
                    db.rollback()  # 发生错误时回滚
                    print(e)
                cursor.close()
